package com.chance.cc.crawler.development.bootstrap.test;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2021/6/1 11:24
 * @Description
 **/
public class FileManager {
    private static Logger log = LoggerFactory.getLogger(FileManager.class);

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    public static void main(String[] args) {
        HttpConfig httpConfig = HttpConfig.me("get");
        httpConfig.setCircularRedirectsAllowed(false);
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);
        httpConfig.setDisableCookie(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i=1;i<=1;i++){
            new Thread(new TestRun(downloader,httpConfig),"thread_"+i).start();
        }


        fileTest();
//        fileManagerRecord();
//        getComment();
//        jsonToExcel();
    }

    static class TestRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            List<String> resultList = new ArrayList<>();
            List<String> list = fileTest();
            for (String url : list) {

//                String url = "http://192.168.1.215:9599/v1/meta/xhs/keys?site=search-high";
//            String url = "https://c.api.weibo.com/2/statuses/count/biz.json?access_token=2.00o4_w1HrAaeYBedf38e38b8SnITmD&ids=4633479504202767";
                HttpRequest httpRequest = new HttpRequest();
//                for (int i = 0; i < 1; i++) {
                    httpRequest.setUrl(url);
                    HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                String s = httpPage.getHtml().xpath("//span[@class=\"post-handle-publish\"]/strong/text() | //div[@class=\"post-site\"]/div[@class=\"post-site-txt\"]/strong/text()|//span[@class=\"publish-time\"]/text()|//div[@class=\"post-info\"]/span[2]/strong/text()").get();
                if(StringUtils.isBlank(s)){
                    System.out.println(url);
                }
//                    List<String> all = httpPage.getJson().jsonPath($_type + ".content").all();

//                    for (String data : all) {
//                        JSONObject jsonObject = JSONObject.parseObject(data);
//                        String keyword = jsonObject.getString("keyword");
//                        String keyword1 = JSONObject.parseObject(keyword).getString("keyword");
                        resultList.add("articleUrl [{"+url+"}],time is [{"+s+"}]");
//                    }
//                    System.out.println(httpPage);
//                }
            }
            dataToFile(resultList);
        }
    }

    private static void dataToFile(List<String> resultList) {
        String filePath = "D:\\Download\\autohome_forum.txt";
        File writeFile = new File(filePath);
        try {
            BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(writeFile), Charset.defaultCharset()));

            for (String data : resultList) {
                bufferedWriter.write(data);
                bufferedWriter.newLine();
                bufferedWriter.flush();
            }
        } catch (Exception e) {
            e.getMessage();
        }
    }


    private static List<String> fileTest() {
        String filePath = "C:\\Users\\Administrator\\Desktop\\autohome.txt";
        List<String> list = new ArrayList<>();
        File writeFile = new File(filePath);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(writeFile), "UTF-8"));

            String s = null;
            while ((s = bufferedReader.readLine()) != null) {
                if (list.contains(s)) {
                    System.out.println(s);
                } else {
                    list.add(s);
                }
            }
        } catch (Exception e) {
            e.getMessage();
        }
        return list;
    }

    private static void fileManagerRecord() {
        String path = "C:\\Users\\Administrator\\Desktop\\dy_video_url.txt";
        String filePath = "D:\\Download\\dy_video_url_json.txt";
        File readFile = new File(path);
        File writeFile = new File(filePath);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(readFile), "UTF-8"));
            BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(writeFile), Charset.defaultCharset()));

            List<Map<String, String>> list = new ArrayList<>();
            String s = null;
            int line = 0;
            Map<String, String> map = new HashMap<>();
            while ((s = bufferedReader.readLine()) != null) {
//                if (StringUtils.isBlank(s)) {
//                    continue;
//                }
//                String videoId = s.substring(s.lastIndexOf("video") + 6, s.lastIndexOf("/?"));
//                map.put(videoId, s);
////                list.add(map);

//                CrawlerData crawlerData = JSONObject.parseObject(s,CrawlerData.class);
//                List<String> keywords = crawlerData.tagsCreator().bizTags().getCategoryTag().getKVTagListVal("keywords");
//                if(keywords != null &&(keywords.get(0).equals("斯凯奇") || keywords.get(0).equals("Skechers"))){
//
//                    System.out.println(line);
//                }
//                line = line + 1;
//                try{
//                    JSONObject.parseObject(s, CrawlerData.class);
////                    bufferedWriter.write(s);
////                    bufferedWriter.newLine();
////                    bufferedWriter.flush();
//                }catch (JSONException e){
//                    s = s.replace("\"}{\"", "\"}\n{\"");
//                    bufferedWriter.write(s);
//                    bufferedWriter.newLine();
//                    bufferedWriter.flush();
//                    System.out.println(line);
//                }
            }
            bufferedWriter.write(JSONObject.toJSONString(map));
            bufferedWriter.newLine();
            bufferedWriter.flush();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void getComment() {
        String path = "C:\\Users\\Administrator\\Desktop\\dy.txt";
        String articlePath = "D:\\Download\\dy_video_json.txt";
        String resultPath = "D:\\Download\\dy_comment.txt";
        File readFile = new File(path);
        File writeFile = new File(resultPath);
        File articleFile = new File(articlePath);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(readFile), "UTF-8"));
            BufferedReader articleReader = new BufferedReader(new InputStreamReader(new FileInputStream(articleFile), "UTF-8"));
            BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(writeFile), Charset.defaultCharset()));

            List<String> dataIdList = new ArrayList<>();
            JSONArray jsonArray = JSONArray.parseArray(articleReader.readLine());
            for (Object o : jsonArray) {
                JSONObject jsonObject = JSONObject.parseObject(String.valueOf(o));
                String videoId = jsonObject.getString("videoId");
                dataIdList.add(videoId);
            }

            String s = null;
            int line = 0;
            while ((s = bufferedReader.readLine()) != null) {
                CrawlerData crawlerData = JSONObject.parseObject(s, CrawlerData.class);
                String parentId = crawlerData.getParentId();
                String[] split = parentId.split("article-");
                if (StringUtils.isNotBlank(parentId) && dataIdList.contains(split[1])) {
                    bufferedWriter.write(s);
                    bufferedWriter.newLine();
                    bufferedWriter.flush();
                    System.out.println(line);
                }
//                line = line + 1;
//                try{
//                    JSONObject.parseObject(s, CrawlerData.class);
////                    bufferedWriter.write(s);
////                    bufferedWriter.newLine();
////                    bufferedWriter.flush();
//                }catch (JSONException e){
//                    s = s.replace("\"}{\"", "\"}\n{\"");
//                    bufferedWriter.write(s);
//                    bufferedWriter.newLine();
//                    bufferedWriter.flush();
//                    System.out.println(line);
//                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

//    private static void jsonToExcel() {
//        String readPath = "D:\\Download\\dy_comment.txt";
//        String dyUrlPath = "D:\\Download\\dy_video_url_json.txt";
//        String writeExcelPath = "D:/chance/dy_result.xlsx";
//        try {
//            BufferedReader dyUrlList = new BufferedReader(new InputStreamReader(new FileInputStream(dyUrlPath), "UTF-8"));
//            Map urlMap = JSONObject.parseObject(dyUrlList.readLine(), Map.class);
//
//            OutputStream fos = new FileOutputStream(writeExcelPath);
//            XSSFWorkbook workBook = new XSSFWorkbook();
//            XSSFSheet sheet = workBook.createSheet();
//
//            XSSFRow row = null;
//            XSSFCell cell = null;
//
//            row = sheet.createRow(0);
//            List<String> list = Arrays.asList("collectionTime", "releaseTime", "dataId","parentId", "data_type", "text", "author_name", "author_id", "author_gender", "author_signature", "author_enterprise_verify_reason", "reply_comment_total","likes","article_url");
//            for (int i = 0; i < list.size(); i++) {
//                cell = row.createCell(i);
//                cell.setCellValue(list.get(i));
//            }
//
//            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(readPath), "UTF-8"));
//
//            String stext = null;
//            int count = 1;
//            while ((stext = bufferedReader.readLine()) != null) {
//                try {
//                    row = sheet.createRow(count);
//                    CrawlerData crawlerData = JSONObject.parseObject(stext, CrawlerData.class);
//                    String parentId = crawlerData.getParentId();
//                    String[] split = parentId.split("article-");
//
//                    Json json = new Json(crawlerData.getCrawlerContent());
//                    cell = row.createCell(0);
//                    cell.setCellValue(String.valueOf(crawlerData.getCollectionTime()));
//                    cell = row.createCell(1);
//                    cell.setCellValue(String.valueOf(crawlerData.getReleaseTime()));
//                    cell = row.createCell(2);
//                    cell.setCellValue(crawlerData.getDataId());
//                    cell = row.createCell(3);
//                    cell.setCellValue(parentId);
//                    cell = row.createCell(4);
//                    cell.setCellValue(crawlerData.tagsCreator().resultTags().getCategoryTag().getLabelTag("comment").getLabel());
//                    cell = row.createCell(5);
//                    cell.setCellValue(json.jsonPath($_type + ".text").get());
//                    cell = row.createCell(6);
//                    cell.setCellValue(json.jsonPath($_type + ".user.nickname").get());
//                    cell = row.createCell(7);
//                    cell.setCellValue(json.jsonPath($_type + ".user.uid").get());
//                    cell = row.createCell(8);
//                    cell.setCellValue(json.jsonPath($_type + ".user.gender").get());
//                    cell = row.createCell(9);
//                    cell.setCellValue(json.jsonPath($_type + ".user.signature").get());
//                    cell = row.createCell(10);
//                    cell.setCellValue(json.jsonPath($_type + ".user.enterprise_verify_reason").get());
//                    cell = row.createCell(11);
//                    cell.setCellValue(json.jsonPath($_type + ".reply_comment_total").get());
//                    cell = row.createCell(12);
//                    cell.setCellValue(json.jsonPath($_type + ".digg_count").get());
//                    cell = row.createCell(13);
//                    cell.setCellValue(String.valueOf(urlMap.get(split[1])));
//
//                    count++;
//                } catch (Exception e) {
//                    log.error(e.getMessage());
//                    log.error("count :" +count);
//                    log.error("message:[{}]",sheet);
//                }
//            }
//            workBook.write(fos);
//            bufferedReader.close();
//        } catch (Exception e) {
//            log.error(e.getMessage());
//        }
//    }
}
